﻿using System;
using System.Collections.Generic;
using System.IO;
using SevenZip;

namespace OpeningBookTokenizer
{
    class Program
    {
        static int ENTRIES_PER_BLOCK = 1024;
        static int BYTES_PER_BLOCK = ENTRIES_PER_BLOCK * sizeof(ulong);
        static int BYTES_PER_SHORT_BLOCK = ENTRIES_PER_BLOCK * sizeof(ushort);

        static int dictionary = 1 << 23;

        // static Int32 posStateBits = 2;
        // static  Int32 litContextBits = 3; // for normal files
        // UInt32 litContextBits = 0; // for 32-bit data
        // static  Int32 litPosBits = 0;
        // UInt32 litPosBits = 2; // for 32-bit data
        // static   Int32 algorithm = 2;
        // static    Int32 numFastBytes = 128;

        static bool eos = false;

        static CoderPropID[] propIDs = 
				{
					CoderPropID.DictionarySize,
					CoderPropID.PosStateBits,
					CoderPropID.LitContextBits,
					CoderPropID.LitPosBits,
					CoderPropID.NumFastBytes,
					CoderPropID.MatchFinder,
					CoderPropID.EndMarker
				};

        // these are the default properties, keeping it simple for now:
        static object[] properties = 
				{
					(Int32)(dictionary),
					(Int32)(4),
					(Int32)(8),
					(Int32)(4),
					(Int32)(273),
					"bt4",
					eos
				};

        static void Main(string[] args)
        {
            SevenZip.Compression.LZMA.Encoder encoder = new SevenZip.Compression.LZMA.Encoder();
            encoder.SetCoderProperties(propIDs, properties);

            MemoryStream outStream2 = new MemoryStream();
            encoder.WriteCoderProperties(outStream2);
            long fileSize = BYTES_PER_BLOCK;
            for (int i = 0; i < 8; i++)
                outStream2.WriteByte((Byte)(fileSize >> (8 * i)));

            byte[] probs = outStream2.ToArray();

            SevenZip.Compression.LZMA.Decoder decoder = new SevenZip.Compression.LZMA.Decoder();
            decoder.SetDecoderProperties(probs);
            int l = sizeof(ulong);
            int sl = sizeof(ushort);

            //FileStream zr = new FileStream("C:\\Users\\steve\\Downloads\\raw\\hashes.z", FileMode.Open);
            //BinaryReader br = new BinaryReader(zr);

            //FileStream uo = new FileStream("C:\\Users\\steve\\Downloads\\raw\\hashes.out", FileMode.Create);
            //BinaryWriter ubr = new BinaryWriter(uo);

            //FileStream tuo = new FileStream("C:\\Users\\steve\\Downloads\\raw\\hashes.out.txt", FileMode.Create);
            //StreamWriter tw = new StreamWriter(tuo);

            //FileStream mzr = new FileStream("C:\\Users\\steve\\Downloads\\raw\\moves.z", FileMode.Open);
            //BinaryReader mbr = new BinaryReader(mzr);

            //FileStream mtuo = new FileStream("C:\\Users\\steve\\Downloads\\raw\\moves.out.txt", FileMode.Create);
            //StreamWriter mtw = new StreamWriter(mtuo);

            //FileStream bs = new FileStream("C:\\Users\\steve\\Downloads\\raw\\obtblock.txt", FileMode.Create);
            //StreamWriter bsw = new StreamWriter(bs);


            //int col = offsets.Length;
            //int coff = 0;
            //for (int blk = 0; blk < col; blk++)
            //{
            //    byte[] zped = new byte[offsets[blk]];
            //    zr.Read(zped, 0, offsets[blk]);
            //    MemoryStream sz = new MemoryStream(zped);
            //    MemoryStream ob = new MemoryStream();
            //    decoder.Code(sz, ob, offsets[blk], lengths[blk], null);
            //    coff += offsets[blk];
            //    byte[] dec = ob.ToArray();
            //    int zoff = 0;
            //    int blkBytes = (lengths[blk] / l);
            //    for (int i = 0; i < blkBytes; i++)
            //    {
            //        ulong d = BitConverter.ToUInt64(dec, zoff);
            //        ubr.Write(d);
            //        tw.Write(Convert.ToString(d) + "\n");
            //        zoff += l;
            //    }

            //    byte[] mzped = new byte[moffsets[blk]];
            //    mzr.Read(mzped, 0, moffsets[blk]);
            //    MemoryStream msz = new MemoryStream(mzped);
            //    MemoryStream mob = new MemoryStream();
            //    decoder.Code(msz, mob, moffsets[blk], mlengths[blk], null);
            //    byte[] mdec = mob.ToArray();
            //    int mzoff = 0;
            //    int mblkBytes = (mlengths[blk] / sl);
            //    for (int i = 0; i < mblkBytes; i++)
            //    {
            //        ushort d = BitConverter.ToUInt16(mdec, mzoff);
            //        mtw.Write(Convert.ToString(d) + "\n");
            //        mzoff += sl;
            //    }
            //}

            //mtw.Close();
            //ubr.Close();
            //tw.Close();

            //return;

            String line;

            StreamReader sr = new StreamReader("hashes");
            StreamReader msr = new StreamReader("moves");

            FileStream book_stream = new FileStream("OpeningBook.cs", FileMode.Create);
            StreamWriter book_writer = new StreamWriter(book_stream);

            MemoryStream mms = new MemoryStream();
            BinaryWriter mbm = new BinaryWriter(mms);

            FileStream msc2 = new FileStream("moves.sob", FileMode.Create);
            BinaryWriter msc = new BinaryWriter(msc2);

            MemoryStream ms = new MemoryStream();
            BinaryWriter bm = new BinaryWriter(ms);

            FileStream sc2 = new FileStream("hashes.sob", FileMode.Create);
            BinaryWriter sc = new BinaryWriter(sc2);

            int offset = 0;
            List<int> offsets = new List<int>();
            List<ulong> lengths = new List<ulong>();
            List<ulong> indexes = new List<ulong>();
            List<int> moffsets = new List<int>();
            List<ulong> mlengths = new List<ulong>();

            ulong idx = 0;
            bool gotIdx = false;
            ulong lastIdx = 0;
            int repeats = 0;
            bool canWrite = false;
            string mline;
            while ((line = sr.ReadLine()) != null)
            {
                mline = msr.ReadLine();
                ulong h = Convert.ToUInt64(line);
                ushort sh = Convert.ToUInt16(mline);

                if ((gotIdx) && (lastIdx == h))
                    repeats++;
                else
                    repeats = 0;

                lastIdx = h;
                if (!gotIdx)
                {
                    gotIdx = true;
                    idx = h;
                }

                if ((canWrite) && (repeats == 0))
                {
                    MemoryStream outStream = new MemoryStream();
                    MemoryStream moutStream = new MemoryStream();

                    mms.Seek(0, SeekOrigin.Begin);
                    ms.Seek(0, SeekOrigin.Begin);

                    encoder.Code(mms, moutStream, -1, -1, null);
                    encoder.Code(ms, outStream, -1, -1, null);

                    int ol = (int)outStream.Length;
                    int mol = (int)moutStream.Length;

                    byte[] bar = outStream.ToArray();
                    byte[] mbar = moutStream.ToArray();
                    indexes.Add(idx);
                    offsets.Add(ol);
                    lengths.Add((ulong)ms.Length);
                    moffsets.Add(mol);
                    mlengths.Add((ulong)mms.Length);

                    msc.Write(mbar, 0, mol);
                    sc.Write(bar, 0, ol);

                    ms.Dispose();
                    bm.Dispose();
                    ms = new MemoryStream();
                    bm = new BinaryWriter(ms);
                    mms.Dispose();
                    mbm.Dispose();
                    mms = new MemoryStream();
                    mbm = new BinaryWriter(mms);

                    gotIdx = true;
                    idx = h;
                    offset = 0;
                    canWrite = false;
                }

                byte[] bc = BitConverter.GetBytes(h);
                byte[] sbc = BitConverter.GetBytes(sh);

                bm.Write(bc, 0, l);
                mbm.Write(sbc, 0, sl);

                offset += l;

                if (offset >= BYTES_PER_BLOCK)
                {
                    canWrite = true;
                }
            }

            if (offset != 0)
            {
                MemoryStream outStream = new MemoryStream();
                MemoryStream moutStream = new MemoryStream();

                mms.Seek(0, SeekOrigin.Begin);
                ms.Seek(0, SeekOrigin.Begin);

                encoder.Code(mms, moutStream, -1, -1, null);
                encoder.Code(ms, outStream, -1, -1, null);

                int ol = (int)outStream.Length;
                int mol = (int)moutStream.Length;

                byte[] bar = outStream.ToArray();
                byte[] mbar = moutStream.ToArray();

                indexes.Add(idx);
                offsets.Add(ol);
                lengths.Add((ulong)ms.Length);

                moffsets.Add(mol);
                mlengths.Add((ulong)mms.Length);

                msc.Write(mbar, 0, mol);
                sc.Write(bar, 0, ol);
            }
            
            //write opeing ook

            string s_probs = string.Join(", ", probs);
            string s_offsets = string.Join(", ", offsets);
            string s_indexes = string.Join(", ", indexes);
            string s_lengths = string.Join(", ", lengths);
            string s_moffsets = string.Join(", ", moffsets);
            string s_mlengths = string.Join(", ", mlengths);

            string mymsg = @"using System;
using System.Collections.Generic;
using System.Text;
using SevenZip;
using System.IO;
using Pwned.Chess.Common;

namespace Pwned.Chess
{
    public static class OpeningBook
    {
        internal static int[] offsets = new int[] { " + s_offsets + @" };
        internal static ulong[] indexes = new ulong[] { " + s_indexes + @" };
        internal static int[] lengths = new int[] { " + s_lengths + @" };
        internal static int[] moffsets = new int[] { " + s_moffsets + @" };
        internal static int[] mlengths = new int[] { " + s_mlengths + @" };

        internal static byte[] props = new byte[] { " + s_probs + @" };

        internal static int ENTRIES_PER_BLOCK = " + ENTRIES_PER_BLOCK.ToString() + @";
        static int dictionary = " + dictionary.ToString() + @";
        static bool eos = " + eos.ToString().ToLower() + @";

        // rest of this is static, not variables substituted.

        static CoderPropID[] propIDs = 
				{
					CoderPropID.DictionarySize,
					CoderPropID.PosStateBits,
					CoderPropID.LitContextBits,
					CoderPropID.LitPosBits,
					CoderPropID.NumFastBytes,
					CoderPropID.MatchFinder,
					CoderPropID.EndMarker
				};

        static object[] properties = 
				{
					(Int32)(dictionary),
					(Int32)(4),
					(Int32)(8),
					(Int32)(4),
					(Int32)(273),
					""bt4"",
					eos
				};

        internal static int BYTES_PER_BLOCK = ENTRIES_PER_BLOCK * sizeof(ulong);

        static Random rnd = new Random();

        internal static SevenZip.Compression.LZMA.Decoder decoder;
        internal static BinaryReader hashReader;
        internal static BinaryReader moveReader;

        public static void Init(string path, IStreamCreator sc)
        {
            SevenZip.Compression.LZMA.Encoder encoder = new SevenZip.Compression.LZMA.Encoder();
            encoder.SetCoderProperties(propIDs, properties);

            decoder = new SevenZip.Compression.LZMA.Decoder();

            MemoryStream outStream2 = new MemoryStream();
            encoder.WriteCoderProperties(outStream2);

            long fileSize = BYTES_PER_BLOCK;
            for (int i = 0; i < 8; i++)
                outStream2.WriteByte((Byte)(fileSize >> (8 * i)));

            byte[] probs = outStream2.ToArray();

            decoder.SetDecoderProperties(probs);

            Stream s = sc.OpenStream(path + ""hashes.sob"");
            hashReader = new BinaryReader(s);

            Stream m = sc.OpenStream(path + ""moves.sob"");
            moveReader = new BinaryReader(m);
        }

        public struct BookBlock
        {
            public int index;
            public byte[] hashes;
            public byte[] moves;
        }

        public static List<BookBlock> bookBlocks = new List<BookBlock>();

        static bool BookBlockCached(int idx, ref BookBlock bb)
        {
            bool res = false;
            int bbc = bookBlocks.Count;
            for (int i = 0; i < bbc; i++)
            {
                if (bookBlocks[i].index == idx)
                {
                    bb = bookBlocks[i];
                    res = true;
                    break;
                }
            }
            return res;
        }

        public static bool GetMove(ulong h, ref Move foundMove, float randomness)
        {
            int f = -1;
            int off = 0;
            int moff = 0;
            for (int i = 0; i < (indexes.Length - 1); i++)
            {
                if ((h >= indexes[i]) && (h < indexes[i + 1]))
                {
                    f = i;
                    break;
                }
                off += offsets[i];
                moff += moffsets[i];
            }
            if (f == -1)
                f = indexes.Length - 1;

            List<int> m = new List<int>();
            List<Move> collectedMoves = new List<Move>();

            BookBlock bb = new BookBlock();
            bool res = false;

            if (BookBlockCached(f, ref bb))
            {
                int zoff = 0;
                int l = sizeof(ulong);
                int blkBytes = (lengths[f] / l);
                for (int i = 0; i < blkBytes; i++)
                {
                    ulong d = BitConverter.ToUInt64(bb.hashes, zoff);
                    if (d == h)
                        m.Add(i);
                    zoff += l;
                }

                int sl = sizeof(ushort);
                foreach (int i in m)
                {
                    int p = i * sl;
                    ushort mm = BitConverter.ToUInt16(bb.moves, p);
                    int from = mm & 63;
                    int to = (mm >> 6) & 63;
                    Move bm = new Move(Util.Square((int)from), Util.Square((int)to));
                    collectedMoves.Add(bm);
                }
            }
            else
            {
                hashReader.BaseStream.Seek(off, SeekOrigin.Begin);
                byte[] zped = new byte[offsets[f]];
                hashReader.Read(zped, 0, offsets[f]);
                bb.index = f;
                MemoryStream sz = new MemoryStream(zped);
                MemoryStream ob = new MemoryStream();

                decoder.Code(sz, ob, offsets[f], lengths[f], null);

                byte[] dec = ob.ToArray();
                bb.hashes = dec;
                int zoff = 0;
                int l = sizeof(ulong);
                int blkBytes = (lengths[f] / l);
                for (int i = 0; i < blkBytes; i++)
                {
                    ulong d = BitConverter.ToUInt64(dec, zoff);
                    if (d == h)
                        m.Add(i);
                    zoff += l;
                }

                moveReader.BaseStream.Seek(moff, SeekOrigin.Begin);
                byte[] mzped = new byte[moffsets[f]];
                moveReader.Read(mzped, 0, moffsets[f]);

                MemoryStream msz = new MemoryStream(mzped);
                MemoryStream mob = new MemoryStream();

                decoder.Code(msz, mob, moffsets[f], mlengths[f], null);

                byte[] mdec = mob.ToArray();
                bb.moves = mdec;
                int sl = sizeof(ushort);
                foreach (int i in m)
                {
                    int p = i * sl;
                    ushort mm = BitConverter.ToUInt16(mdec, p);
                    int from = mm & 63;
                    int to = (mm >> 6) & 63;
                    Move bm = new Move(Util.Square((int)from), Util.Square((int)to));
                    collectedMoves.Add(bm);
                }
                if (bookBlocks.Count > 9)
                    bookBlocks.RemoveAt(0);

                bookBlocks.Add(bb);
            }

            if (collectedMoves.Count > 0)
            {
                foundMove = collectedMoves[(int)(randomness * rnd.Next(collectedMoves.Count))];
                res = true;
            }

            return res;
        }
    }
}
";

            book_writer.Write(mymsg);
            //public static int GetScore(int index)
            //{
            //    //ushort i = moves[index];
            //    //int score = (i >> 12) & 31;

            //    //return score;
            //    return 0;
            //}
            book_writer.Close();
            book_stream.Close();
            msc.Close();
            msr.Close();
            sc.Close();
            sr.Close();
        }
    }
}
